#ifndef xfifo_clean_dcache
#define xfifo_clean_dcache(addr, size)          do {} while (0)
#endif
#ifndef xfifo_invalidate_dcache
#define xfifo_invalidate_dcache(addr, size)     do {} while (0)
#endif

#ifndef offsetof
#define offsetof(TYPE, MEMBER)                  ((size_t)&((TYPE *)0)->MEMBER)
#endif

#ifndef READ_ONCE
#define READ_ONCE(x)                            (x)
#endif

#ifndef WRITE_ONCE
#define WRITE_ONCE(x, val)                      ({ (x) = (val); })
#endif

#if ENABLE_CROSS_DEVICE_BUILD
/* static check at compile time */
static_assert(offsetof(struct __xfifo, in) / SHM_CACHELINE_BYTES != offsetof(struct __xfifo, out) / SHM_CACHELINE_BYTES,
              "not match");
#endif

__static_inline
int XFIFO_FUNC(initialized)(struct __xfifo *fifo)
{
    return fifo->mask && ((fifo->mask + 1) & fifo->mask) == 0;
}

/**
 * fifo_init - initialize a fifo
 * @fifo:	the fifo to be initialized
 * @data:	the data buffer for the fifo
 * @size:	the size of the data buffer in bytes
 * @esize:	the size of each element in the fifo in bytes
 *
 * Returns 0 on success, negative error code on failure.
 */
__static_inline
int XFIFO_FUNC(init)(struct __xfifo *fifo,
                     void *data, uint32_t size, uint32_t esize)
{
    if (size < 2) {
        return -EINVAL;
    }

    if (size % esize) {
        return -EINVAL;
    }

    /* size is not 2^n ? */
    if (size & (size - 1)) {
        return -EINVAL;
    }

#if ENABLE_CROSS_DEVICE_BUILD
    {
        int32_t offset = (long)(void *)data - (long)(void *)fifo;

        if (READ_ONCE(fifo->mask) == size - 1
                && READ_ONCE(fifo->data_offset) == offset) {
            return -EBUSY;
        }

        WRITE_ONCE(fifo->data_offset, offset);
    }
#else
    if (READ_ONCE(fifo->mask) == size - 1
            && READ_ONCE(fifo->data) == data) {
        return -EBUSY;
    }

    WRITE_ONCE(fifo->data, data);
#endif

    WRITE_ONCE(fifo->in, 0);
    WRITE_ONCE(fifo->out, 0);
    WRITE_ONCE(fifo->mask, size - 1);
    WRITE_ONCE(fifo->esize, esize);

    xfifo_clean_dcache(fifo, sizeof(*fifo));

    return 0;
}

__static_inline
uint32_t XFIFO_FUNC(size)(struct __xfifo *fifo)
{
    return fifo->mask + 1;
}

__static_inline
uint32_t XFIFO_FUNC(used_len)(struct __xfifo *fifo)
{
    xfifo_invalidate_dcache(&fifo->in, sizeof(fifo->in));
    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));

    return READ_ONCE(fifo->in) - READ_ONCE(fifo->out);
}

__static_inline
uint32_t XFIFO_FUNC(unused_len)(struct __xfifo *fifo)
{
    xfifo_invalidate_dcache(&fifo->in, sizeof(fifo->in));
    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));

    return (fifo->mask + 1) - (READ_ONCE(fifo->in) - READ_ONCE(fifo->out));
}

__static_inline
int XFIFO_FUNC(is_empty)(struct __xfifo *fifo)
{
    xfifo_invalidate_dcache(&fifo->in, sizeof(fifo->in));
    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));

    return READ_ONCE(fifo->in) == READ_ONCE(fifo->out);
}

__static_inline
int XFIFO_FUNC(is_full)(struct __xfifo *fifo)
{
    return XFIFO_FUNC(used_len(fifo)) > fifo->mask;
}

__static_inline
void *XFIFO_FUNC(data)(struct __xfifo *fifo, uint32_t offset)
{
#if ENABLE_CROSS_DEVICE_BUILD
    return (void *)fifo + fifo->data_offset + (offset & fifo->mask);
#else
    return fifo->data + (offset & fifo->mask);
#endif
}

__static_inline
void *XFIFO_FUNC(data_mem)(struct __xfifo *fifo)
{
    if (XFIFO_FUNC(is_empty)(fifo)) {
        return NULL;
    }

    return XFIFO_FUNC(data)(fifo, READ_ONCE(fifo->out));
}

__static_inline
int XFIFO_FUNC(enqueue_len)(struct __xfifo *fifo, void *data, uint32_t len)
{
    void *pos;
    uint32_t in;

    if (XFIFO_FUNC(unused_len)(fifo) < len) {
        return -EBUSY;
    }

    xfifo_invalidate_dcache(&fifo->in, sizeof(fifo->in));
    in = READ_ONCE(fifo->in);
    pos = XFIFO_FUNC(data)(fifo, in);
    memcpy(pos, data, len);
    WRITE_ONCE(fifo->in, in + len);
    xfifo_clean_dcache(pos, len);
    xfifo_clean_dcache(&fifo->in, sizeof(fifo->in));

    return 0;
}

__static_inline
int XFIFO_FUNC(skip_len)(struct __xfifo *fifo, uint32_t len);

__static_inline
int XFIFO_FUNC(enqueue_overlay_len)(struct __xfifo *fifo, void *data, uint32_t len)
{
    void *pos;
    uint32_t in;

    if (XFIFO_FUNC(unused_len)(fifo) < len) {
        XFIFO_FUNC(skip_len)(fifo, len);
    }

    xfifo_invalidate_dcache(&fifo->in, sizeof(fifo->in));
    in = READ_ONCE(fifo->in);
    pos = XFIFO_FUNC(data)(fifo, in);
    memcpy(pos, data, len);
    WRITE_ONCE(fifo->in, in + len);
    xfifo_clean_dcache(pos, len);
    xfifo_clean_dcache(&fifo->in, sizeof(fifo->in));

    return 0;
}

__static_inline
int XFIFO_FUNC(dequeue_len)(struct __xfifo *fifo, void *dest, uint32_t len)
{
    void *pos;
    uint32_t out;

    if (XFIFO_FUNC(used_len)(fifo) < len) {
        return -ENXIO;
    }

    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));
    out = READ_ONCE(fifo->out);
    pos = XFIFO_FUNC(data)(fifo, out);
    xfifo_invalidate_dcache(pos, len);
    memcpy(dest, pos, len);
    WRITE_ONCE(fifo->out, out + len);
    xfifo_clean_dcache(&fifo->out, sizeof(fifo->out));

    return 0;
}

/**
 * __xfifo_peek_len - get data from the fifo without removing
 */
__static_inline
int XFIFO_FUNC(peek_len)(struct __xfifo *fifo, void *dest, uint32_t len)
{
    void *pos;

    if (XFIFO_FUNC(used_len)(fifo) < len) {
        return -ENXIO;
    }

    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));
    pos = XFIFO_FUNC(data)(fifo, READ_ONCE(fifo->out));
    xfifo_invalidate_dcache(pos, len);
    memcpy(dest, pos, len);

    return 0;
}

/**
 * __xfifo_skip_len - skip/remove output data
 */
__static_inline
int XFIFO_FUNC(skip_len)(struct __xfifo *fifo, uint32_t len)
{
    if (XFIFO_FUNC(used_len)(fifo) < len) {
        return -ENXIO;
    }

    xfifo_invalidate_dcache(&fifo->out, sizeof(fifo->out));
    WRITE_ONCE(fifo->out, READ_ONCE(fifo->out) + len);
    xfifo_clean_dcache(&fifo->out, sizeof(fifo->out));

    return 0;
}

__static_inline
uint32_t XFIFO_FUNC(used_count)(struct __xfifo *fifo)
{
    return XFIFO_FUNC(used_len)(fifo) / fifo->esize;
}

__static_inline
uint32_t XFIFO_FUNC(unused_count)(struct __xfifo *fifo)
{
    return XFIFO_FUNC(unused_len)(fifo) / fifo->esize;
}

__static_inline
int XFIFO_FUNC(enqueue)(struct __xfifo *fifo, void *data)
{
    return XFIFO_FUNC(enqueue_len)(fifo, data, fifo->esize);
}

__static_inline
int XFIFO_FUNC(enqueue_overlay)(struct __xfifo *fifo, void *data)
{
    return XFIFO_FUNC(enqueue_overlay_len)(fifo, data, fifo->esize);
}

__static_inline
int XFIFO_FUNC(dequeue)(struct __xfifo *fifo, void *dest)
{
    return XFIFO_FUNC(dequeue_len)(fifo, dest, fifo->esize);
}

/**
 * __xfifo_peek - get data from the fifo without removing
 */
__static_inline
int XFIFO_FUNC(peek)(struct __xfifo *fifo, void *dest)
{
    return XFIFO_FUNC(peek_len)(fifo, dest, fifo->esize);
}

/**
 * __xfifo_skip - skip/remove output data
 */
__static_inline
int XFIFO_FUNC(skip)(struct __xfifo *fifo)
{
    return XFIFO_FUNC(skip_len)(fifo, fifo->esize);
}
